/*
 * University Of Washington - Tacoma
 * TCSS-422, Operating Systems
 * Assignment 1 - Web Crawler
 * April 26, 2011
 * 
 * Team: Kernel Monkeys
 * Andrew Boguk
 * Sergiy Kulyk
 * Nicholas Swartzendruber
 */

package crawler.engine;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;
import java.util.Observable;
import java.util.concurrent.BlockingQueue;

/**
 * Retrieve webpages, does some housekeeping on the
 * info and then puts it into a thread-safe queue
 * to be serviced.
 * 
 * @author Sergiy Kulyk
 * @version April 24, 2011
 */
public class PageRetriever extends Observable implements Runnable {
	
	/**
	 * End of line character. It is used to split the byte array into
	 * lines of text.
	 */
	private static final String LINE_END = "\n";

	/**
	 * The blocking queue where to store retrieved pages.
	 */
	private final BlockingQueue<List<String>> my_page_q;
	
	/**
	 * The web page url from where to retrieve the page.
	 */
	private final URL my_url;
	
	/**
	 * Constructor.
	 * 
	 * @param the_url The URL this retriever is supposed to get.
	 * @param the_page_buffer The queue where to place the downloaded page.
	 */
	public PageRetriever(final URL the_url, 
						 final BlockingQueue<List<String>> the_page_buffer) {
		
		my_page_q = the_page_buffer;
		my_url = the_url;		
	}

	
	/**
	 * This method starts the Retriever. It will be called automatically
	 * by the Thread class. After it is called, the Retriever will
	 * download the page according to the provided URL, and store
	 * it in the Queue.
	 * 
	 * {@link} http://nadeausoftware.com/node/73
	 */
	@Override
	public void run() {
		
		try {
			final java.net.URLConnection connection = my_url.openConnection( );
	        if ( !(connection instanceof java.net.HttpURLConnection) )
	            throw new java.lang.IllegalArgumentException(
	                "URL protocol must be HTTP." );
	        final java.net.HttpURLConnection conn =
	            (java.net.HttpURLConnection)connection;
	 
	        // Set up a request.
	        conn.setConnectTimeout( 10000 );    // 10 sec
	        conn.setReadTimeout( 10000 );       // 10 sec
	        conn.setInstanceFollowRedirects( true );
	        conn.setRequestProperty( "User-agent", "TCSS-422 spider" );
	 
	        // Send the request.
	        conn.connect( );
	        final int length  = conn.getContentLength( );
	 
	        // Get the content.
	        byte[] content = null;
	        
	        final InputStream stream = conn.getInputStream();
	        if ( stream != null ) {	            
					content = readStream( length, stream );				
	        } 
	        
	        conn.disconnect( );
	        
	        if (content != null) {
		        
	        	my_page_q.put(bytesToStrings("http://" + my_url.getHost() + my_url.getPath(), content));
	        }	        
	        
		} catch (final IOException e) {
			// skip the page if cannot connect to it			
		} catch (final InterruptedException e) {
			
		}
        
        setChanged();
		notifyObservers();
	}
	
	
	/**
	 * Form a list of strings out of a byte array. The first string
	 * is a URL of the page. 
	 * 
	 * @param the_url Page URL.
	 * @param the_content Page content on its byte form.
	 * @return Page content in a form of a list of strings.
	 */
	private List<String> bytesToStrings(final String the_url, final byte[] the_content) {
		
		final List<String> lines = new LinkedList<String>();
		final String[] page_array = (new String(the_content)).split(LINE_END);
		
		lines.add(the_url);
		
		for (String line : page_array) {
			lines.add(line);
		}
		
		return lines;
	}


	/**
	 * Read the stream of bytes.
	 * 
	 * @param length Expected length of the content.
	 * @param stream The stream.
	 * @return Bytes read from the stream.
	 * @throws java.io.IOException Throws an exception 
	 *         if it is unable to read from the stream.
	 */
    protected byte[] readStream(final int length, final InputStream stream )
        throws java.io.IOException {
        final int buflen = 
        	Math.max( 1024, Math.max( length, stream.available() ) );
        byte[] buf   = new byte[buflen];;
        byte[] bytes = null;
 
        for ( int nRead = stream.read(buf); nRead != -1; nRead = stream.read(buf) ) {
            if ( bytes == null ) {
                bytes = buf;
                buf   = new byte[buflen];
                continue;
            }
            final byte[] newBytes = new byte[ bytes.length + nRead ];
            System.arraycopy( bytes, 0, newBytes, 0, bytes.length );
            System.arraycopy( buf, 0, newBytes, bytes.length, nRead );
            bytes = newBytes;
        }
        
        return bytes;
    }
}